{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import sys      ###policyevaluator4header\n",
    "sys.path.append('../scripts/')\n",
    "from puddle_world import *\n",
    "import itertools \n",
    "import collections #追加"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "class PolicyEvaluator:   ###policyevaluator4\n",
    "    def __init__(self, widths, goal, time_interval, sampling_num, lowerleft=np.array([-4, -4]).T, upperright=np.array([4, 4]).T): #引数追加\n",
    "        self.pose_min = np.r_[lowerleft, 0]\n",
    "        self.pose_max = np.r_[upperright, math.pi*2]\n",
    "        self.widths = widths\n",
    "        self.goal = goal\n",
    "        \n",
    "        self.index_nums = ((self.pose_max - self.pose_min)/self.widths).astype(int)\n",
    "        nx, ny, nt = self.index_nums\n",
    "        self.indexes = list(itertools.product(range(nx), range(ny), range(nt)))\n",
    "        \n",
    "        self.value_function, self.final_state_flags =  self.init_value_function() \n",
    "        self.policy = self.init_policy()\n",
    "        self.actions = list(set([tuple(self.policy[i]) for i in self.indexes])) #追加（policyの行動をsetにすることで重複削除し、リスト化）\n",
    "        \n",
    "        self.state_transition_probs = self.init_state_transition_probs(time_interval, sampling_num) #追加\n",
    "    \n",
    "    def init_state_transition_probs(self, time_interval, sampling_num): #追加\n",
    "        ###セルの中の座標を均等にsampling_num**3点サンプリング###\n",
    "        dx = np.linspace(0.001, self.widths[0]*0.999, sampling_num) #隣のセルにはみ出さないように端を避ける\n",
    "        dy = np.linspace(0.001, self.widths[1]*0.999, sampling_num)\n",
    "        dt = np.linspace(0.001, self.widths[2]*0.999, sampling_num)\n",
    "        samples = list(itertools.product(dx, dy, dt))\n",
    "        \n",
    "        ###各行動、各方角でサンプリングした点を移動してインデックスの増分を記録###\n",
    "        tmp = {}\n",
    "        for a in self.actions:\n",
    "            for i_t in range(self.index_nums[2]):\n",
    "                transitions = []\n",
    "                for s in samples:\n",
    "                    before = np.array([s[0], s[1], s[2] + i_t*self.widths[2]]).T + self.pose_min  #遷移前の姿勢\n",
    "                    before_index = np.array([0, 0, i_t]).T                                                      #遷移前のインデックス\n",
    "                \n",
    "                    after = IdealRobot.state_transition(a[0], a[1], time_interval, before)   #遷移後の姿勢\n",
    "                    after_index = np.floor((after - self.pose_min)/self.widths).astype(int)   #遷移後のインデックス\n",
    "                    \n",
    "                    transitions.append(after_index - before_index)                                  #インデックスの差分を追加\n",
    "                    \n",
    "                unique, count = np.unique(transitions, axis=0, return_counts=True)   #集計（どのセルへの遷移が何回か）\n",
    "                probs = [c/sampling_num**3 for c in count]                   #サンプル数で割って確率にする\n",
    "                tmp[a,i_t] = list(zip(unique, probs))\n",
    "                \n",
    "        return tmp\n",
    "        \n",
    "    def init_policy(self):\n",
    "        tmp = np.zeros(np.r_[self.index_nums,2]) #制御出力が2次元なので、配列の次元を4次元に\n",
    "        for index in self.indexes:\n",
    "            center = self.pose_min + self.widths*(np.array(index).T + 0.5)  #セルの中心の座標\n",
    "            tmp[index] = PuddleIgnoreAgent.policy(center, self.goal)\n",
    "            \n",
    "        return tmp\n",
    "        \n",
    "    def init_value_function(self): \n",
    "        v = np.empty(self.index_nums) #全離散状態を要素に持つ配列を作成\n",
    "        f = np.zeros(self.index_nums) \n",
    "        \n",
    "        for index in self.indexes:\n",
    "            f[index] = self.final_state(np.array(index).T)\n",
    "            v[index] = self.goal.value if f[index] else -100.0\n",
    "                \n",
    "        return v, f\n",
    "        \n",
    "    def final_state(self, index):\n",
    "        x_min, y_min, _ = self.pose_min + self.widths*index          #xy平面で左下の座標\n",
    "        x_max, y_max, _ = self.pose_min + self.widths*(index + 1) #右上の座標（斜め上の離散状態の左下の座標）\n",
    "        \n",
    "        corners = [[x_min, y_min, _], [x_min, y_max, _], [x_max, y_min, _], [x_max, y_max, _] ] #4隅の座標\n",
    "        return all([self.goal.inside(np.array(c).T) for c in corners ])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "{((0.0, -2.0), 0): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 1): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 2): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 3): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 4): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 5): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 6): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 7): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 8): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 9): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 10): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 11): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 12): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 13): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 14): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 15): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 16): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 17): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 18): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 19): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 20): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 21): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 22): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 23): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 24): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 25): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 26): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 27): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 28): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 29): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 30): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 31): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 32): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 33): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 34): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, -2.0), 35): [(array([ 0,  0, -2]), 0.20000000000000001),\n",
       "  (array([ 0,  0, -1]), 0.80000000000000004)],\n",
       " ((0.0, 2.0), 0): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 1): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 2): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 3): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 4): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 5): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 6): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 7): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 8): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 9): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 10): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 11): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 12): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 13): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 14): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 15): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 16): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 17): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 18): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 19): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 20): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 21): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 22): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 23): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 24): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 25): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 26): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 27): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 28): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 29): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 30): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 31): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 32): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 33): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 34): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((0.0, 2.0), 35): [(array([0, 0, 1]), 0.80000000000000004),\n",
       "  (array([0, 0, 2]), 0.20000000000000001)],\n",
       " ((1.0, 0.0), 0): [(array([0, 0, 0]), 0.45500000000000002),\n",
       "  (array([0, 1, 0]), 0.044999999999999998),\n",
       "  (array([1, 0, 0]), 0.45500000000000002),\n",
       "  (array([1, 1, 0]), 0.044999999999999998)],\n",
       " ((1.0, 0.0), 1): [(array([0, 0, 0]), 0.41499999999999998),\n",
       "  (array([0, 1, 0]), 0.085000000000000006),\n",
       "  (array([1, 0, 0]), 0.41499999999999998),\n",
       "  (array([1, 1, 0]), 0.085000000000000006)],\n",
       " ((1.0, 0.0), 2): [(array([0, 0, 0]), 0.40100000000000002),\n",
       "  (array([0, 1, 0]), 0.129),\n",
       "  (array([1, 0, 0]), 0.35899999999999999),\n",
       "  (array([1, 1, 0]), 0.111)],\n",
       " ((1.0, 0.0), 3): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([0, 1, 0]), 0.17999999999999999),\n",
       "  (array([1, 0, 0]), 0.28000000000000003),\n",
       "  (array([1, 1, 0]), 0.12)],\n",
       " ((1.0, 0.0), 4): [(array([0, 0, 0]), 0.38400000000000001),\n",
       "  (array([0, 1, 0]), 0.23599999999999999),\n",
       "  (array([1, 0, 0]), 0.23599999999999999),\n",
       "  (array([1, 1, 0]), 0.14399999999999999)],\n",
       " ((1.0, 0.0), 5): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([0, 1, 0]), 0.28000000000000003),\n",
       "  (array([1, 0, 0]), 0.17999999999999999),\n",
       "  (array([1, 1, 0]), 0.12)],\n",
       " ((1.0, 0.0), 6): [(array([0, 0, 0]), 0.40100000000000002),\n",
       "  (array([0, 1, 0]), 0.35899999999999999),\n",
       "  (array([1, 0, 0]), 0.129),\n",
       "  (array([1, 1, 0]), 0.111)],\n",
       " ((1.0, 0.0), 7): [(array([0, 0, 0]), 0.41499999999999998),\n",
       "  (array([0, 1, 0]), 0.41499999999999998),\n",
       "  (array([1, 0, 0]), 0.085000000000000006),\n",
       "  (array([1, 1, 0]), 0.085000000000000006)],\n",
       " ((1.0, 0.0), 8): [(array([0, 0, 0]), 0.45500000000000002),\n",
       "  (array([0, 1, 0]), 0.45500000000000002),\n",
       "  (array([1, 0, 0]), 0.044999999999999998),\n",
       "  (array([1, 1, 0]), 0.044999999999999998)],\n",
       " ((1.0, 0.0), 9): [(array([0, 0, 0]), 0.45500000000000002),\n",
       "  (array([0, 1, 0]), 0.45500000000000002),\n",
       "  (array([-1,  0,  0]), 0.044999999999999998),\n",
       "  (array([-1,  1,  0]), 0.044999999999999998)],\n",
       " ((1.0, 0.0), 10): [(array([0, 0, 0]), 0.41499999999999998),\n",
       "  (array([0, 1, 0]), 0.41499999999999998),\n",
       "  (array([-1,  0,  0]), 0.085000000000000006),\n",
       "  (array([-1,  1,  0]), 0.085000000000000006)],\n",
       " ((1.0, 0.0), 11): [(array([0, 0, 0]), 0.40600000000000003),\n",
       "  (array([0, 1, 0]), 0.36399999999999999),\n",
       "  (array([-1,  0,  0]), 0.124),\n",
       "  (array([-1,  1,  0]), 0.106)],\n",
       " ((1.0, 0.0), 12): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([0, 1, 0]), 0.28000000000000003),\n",
       "  (array([-1,  0,  0]), 0.17999999999999999),\n",
       "  (array([-1,  1,  0]), 0.12)],\n",
       " ((1.0, 0.0), 13): [(array([0, 0, 0]), 0.39000000000000001),\n",
       "  (array([0, 1, 0]), 0.23999999999999999),\n",
       "  (array([-1,  0,  0]), 0.23000000000000001),\n",
       "  (array([-1,  1,  0]), 0.14000000000000001)],\n",
       " ((1.0, 0.0), 14): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([0, 1, 0]), 0.17999999999999999),\n",
       "  (array([-1,  0,  0]), 0.28000000000000003),\n",
       "  (array([-1,  1,  0]), 0.12)],\n",
       " ((1.0, 0.0), 15): [(array([0, 0, 0]), 0.40100000000000002),\n",
       "  (array([0, 1, 0]), 0.129),\n",
       "  (array([-1,  0,  0]), 0.35899999999999999),\n",
       "  (array([-1,  1,  0]), 0.111)],\n",
       " ((1.0, 0.0), 16): [(array([0, 0, 0]), 0.41499999999999998),\n",
       "  (array([0, 1, 0]), 0.085000000000000006),\n",
       "  (array([-1,  0,  0]), 0.41499999999999998),\n",
       "  (array([-1,  1,  0]), 0.085000000000000006)],\n",
       " ((1.0, 0.0), 17): [(array([0, 0, 0]), 0.45500000000000002),\n",
       "  (array([0, 1, 0]), 0.044999999999999998),\n",
       "  (array([-1,  0,  0]), 0.45500000000000002),\n",
       "  (array([-1,  1,  0]), 0.044999999999999998)],\n",
       " ((1.0, 0.0), 18): [(array([0, 0, 0]), 0.45500000000000002),\n",
       "  (array([ 0, -1,  0]), 0.044999999999999998),\n",
       "  (array([-1,  0,  0]), 0.45500000000000002),\n",
       "  (array([-1, -1,  0]), 0.044999999999999998)],\n",
       " ((1.0, 0.0), 19): [(array([0, 0, 0]), 0.41499999999999998),\n",
       "  (array([ 0, -1,  0]), 0.085000000000000006),\n",
       "  (array([-1,  0,  0]), 0.41499999999999998),\n",
       "  (array([-1, -1,  0]), 0.085000000000000006)],\n",
       " ((1.0, 0.0), 20): [(array([0, 0, 0]), 0.41399999999999998),\n",
       "  (array([ 0, -1,  0]), 0.126),\n",
       "  (array([-1,  0,  0]), 0.35599999999999998),\n",
       "  (array([-1, -1,  0]), 0.104)],\n",
       " ((1.0, 0.0), 21): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([ 0, -1,  0]), 0.17999999999999999),\n",
       "  (array([-1,  0,  0]), 0.28000000000000003),\n",
       "  (array([-1, -1,  0]), 0.12)],\n",
       " ((1.0, 0.0), 22): [(array([0, 0, 0]), 0.39600000000000002),\n",
       "  (array([ 0, -1,  0]), 0.23400000000000001),\n",
       "  (array([-1,  0,  0]), 0.23400000000000001),\n",
       "  (array([-1, -1,  0]), 0.13600000000000001)],\n",
       " ((1.0, 0.0), 23): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([ 0, -1,  0]), 0.28000000000000003),\n",
       "  (array([-1,  0,  0]), 0.17999999999999999),\n",
       "  (array([-1, -1,  0]), 0.12)],\n",
       " ((1.0, 0.0), 24): [(array([0, 0, 0]), 0.40600000000000003),\n",
       "  (array([ 0, -1,  0]), 0.36399999999999999),\n",
       "  (array([-1,  0,  0]), 0.124),\n",
       "  (array([-1, -1,  0]), 0.106)],\n",
       " ((1.0, 0.0), 25): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([ 0, -1,  0]), 0.41999999999999998),\n",
       "  (array([-1,  0,  0]), 0.080000000000000002),\n",
       "  (array([-1, -1,  0]), 0.080000000000000002)],\n",
       " ((1.0, 0.0), 26): [(array([0, 0, 0]), 0.45500000000000002),\n",
       "  (array([ 0, -1,  0]), 0.45500000000000002),\n",
       "  (array([-1,  0,  0]), 0.044999999999999998),\n",
       "  (array([-1, -1,  0]), 0.044999999999999998)],\n",
       " ((1.0, 0.0), 27): [(array([0, 0, 0]), 0.45500000000000002),\n",
       "  (array([ 0, -1,  0]), 0.45500000000000002),\n",
       "  (array([1, 0, 0]), 0.044999999999999998),\n",
       "  (array([ 1, -1,  0]), 0.044999999999999998)],\n",
       " ((1.0, 0.0), 28): [(array([0, 0, 0]), 0.41499999999999998),\n",
       "  (array([ 0, -1,  0]), 0.41499999999999998),\n",
       "  (array([1, 0, 0]), 0.085000000000000006),\n",
       "  (array([ 1, -1,  0]), 0.085000000000000006)],\n",
       " ((1.0, 0.0), 29): [(array([0, 0, 0]), 0.40799999999999997),\n",
       "  (array([ 0, -1,  0]), 0.35199999999999998),\n",
       "  (array([1, 0, 0]), 0.13200000000000001),\n",
       "  (array([ 1, -1,  0]), 0.108)],\n",
       " ((1.0, 0.0), 30): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([ 0, -1,  0]), 0.28000000000000003),\n",
       "  (array([1, 0, 0]), 0.17999999999999999),\n",
       "  (array([ 1, -1,  0]), 0.12)],\n",
       " ((1.0, 0.0), 31): [(array([0, 0, 0]), 0.39000000000000001),\n",
       "  (array([ 0, -1,  0]), 0.23000000000000001),\n",
       "  (array([1, 0, 0]), 0.23999999999999999),\n",
       "  (array([ 1, -1,  0]), 0.14000000000000001)],\n",
       " ((1.0, 0.0), 32): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([ 0, -1,  0]), 0.17999999999999999),\n",
       "  (array([1, 0, 0]), 0.28000000000000003),\n",
       "  (array([ 1, -1,  0]), 0.12)],\n",
       " ((1.0, 0.0), 33): [(array([0, 0, 0]), 0.40600000000000003),\n",
       "  (array([ 0, -1,  0]), 0.124),\n",
       "  (array([1, 0, 0]), 0.36399999999999999),\n",
       "  (array([ 1, -1,  0]), 0.106)],\n",
       " ((1.0, 0.0), 34): [(array([0, 0, 0]), 0.41999999999999998),\n",
       "  (array([ 0, -1,  0]), 0.080000000000000002),\n",
       "  (array([1, 0, 0]), 0.41999999999999998),\n",
       "  (array([ 1, -1,  0]), 0.080000000000000002)],\n",
       " ((1.0, 0.0), 35): [(array([0, 0, 0]), 0.45500000000000002),\n",
       "  (array([ 0, -1,  0]), 0.044999999999999998),\n",
       "  (array([1, 0, 0]), 0.45500000000000002),\n",
       "  (array([ 1, -1,  0]), 0.044999999999999998)]}"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pe = PolicyEvaluator(np.array([0.2, 0.2, math.pi/18]).T, Goal(-3,-3), 0.1, 10)   ##policyevaluator4exec\n",
    "pe.state_transition_probs"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
